import scrapy

from scrapy.selector import HtmlXPathSelector

from movie.items import MovieItem

class LoldyttSpider(scrapy.Spider):
    name = "loldytt"
    allowed_domains = ["www.loldytt.com", "loldytt.com", "img.aolusb.com"]
    start_urls = [
        "http://www.loldytt.com/",

    ]

    def parse(self, response):
        filename = "test.html"
        open(filename, 'wb').write(response.body)
        items = []
        sel = HtmlXPathSelector(response)
        data = sel.select('//div[@class="new_nav"]//li')
        for site in data:
            item = MovieItem()
            item['title'] = site.select('a/text()').extract()
            item['url'] = site.select('a/text()').extract()
            items.append(item)
        return items